#!/usr/bin/env bash

CALL_FORMAT="$0 <url_list> <output> [<lang> <project, wiki - for wikipedia, any for any>]"
URL_LIST=$1
OUTPUT_DIR=$2
LANG=$3
PROJECT=$4

if [ -z "$URL_LIST" ];
then
    echo "$CALL_FORMAT" >&2
    exit
fi

if [ -z "$OUTPUT_DIR" ];
then
    echo "$CALL_FORMAT" >&2
    exit
fi

if [ -z "$PROJECT" ];
then
    PROJECT=any
fi


if [ -n "$LANG" ];
then
    FILTER=^$LANG
else
    FILTER=^[a-z][a-z]
fi

if [ "$PROJECT" == "any" ];
then
    FILTER=$FILTER
elif [ "$PROJECT" == "wiki" ];
then
    FILTER=$FILTER" "
else
    FILTER=$FILTER"."$PROJECT
fi


cat "$URL_LIST" |
while read -r URL;
do
    OUTPUT=$OUTPUT_DIR/`basename "$URL"`

    TIMES_TRIED=0
    MAX_TRIES=10
    IS_SUCCESS=''

    while [ -z "$IS_SUCCESS" -a "$TIMES_TRIED" -lt "$MAX_TRIES" ];
    do
        STATUS=0
        echo "downloading \"$URL\""
        echo '' |
            wget --tries=1 "$URL" --output-document=- --no-verbose |
            gunzip |
            grep "$FILTER" |
            cut -f 1,2,3 -d ' ' |
            gzip > $OUTPUT

        for SP in "${PIPESTATUS[@]}"
        do
            if [ "$SP" -ne "0" ];
            then
                STATUS=1
            fi
        done

        if [ "$STATUS" -ne "0" ];
        then
            IS_SUCCESS=''
            TIMES_TRIED=$((TIMES_TRIED + 1))
            echo "error downloading \"$URL\", try=$TIMES_TRIED"
        else
            IS_SUCCESS=true
        fi

    done

    if [ -z "$IS_SUCCESS" ];
    then
        rm -rf "$OUTPUT"
    fi
done
